In [119]:
# multiple time series analysis
# 20160724, 20160801
# Sean Chang
In [44]:
import numpy as np
import pandas as pd
import scipy as sp
import os
import matplotlib.pyplot as plt
%matplotlib inline
import seaborn as sns
sns.set(style="darkgrid")
from datetime import datetime
import matplotlib.pyplot as plt
#os.chdir('/home/sc268/Dropbox/codes/2016_timeseries_currency/')
#print os.getcwd()
In [ ]:
## UDFs
def merge_ts(all_ts):
all_ts_arr = []
ts_id = []
for id in all_ts.keys():
all_ts_arr.append(all_ts[id])
ts_id.append(id)
df = pd.concat(all_ts_arr, join='outer', axis=1)
df.columns = ts_id
return df
def to_percentage_change(all_ts):
normalized_ts = {}
for id in all_ts.keys():
ts = all_ts[id]
normalized_ts[id] = (ts - ts.shift()) / ts
return normalized_ts
def plot_corr(all_ts):
# merge time series with shared timestamps (dates)
df = merge_ts(all_ts)
corr_mat = df.corr()
sm.graphics.plot_corr(corr_mat, xnames=df.columns)
return corr_mat
def clean_up_df(dat):
print dat.head()
# delete unnecessary columns
dat = dat.ix[:, 0:4]
# filter:: only consider 8 am in each day
dat = dat.loc[[time == 80000 for time in dat['<TIME>']]].reset_index(drop=True)
# create timestamp
dat['time'] = map(lambda dd, tt: datetime.strptime(str(dd)+str(tt),'%Y%m%d%H%M%S'), dat['<DTYYYYMMDD>'], dat['<TIME>'])
# rename
dat.rename(columns={'<TICKER>':'currency', '<OPEN>':'rate'}, inplace=True)
# delete unused columns
del dat['<DTYYYYMMDD>'], dat['<TIME>']
print dat.head()
return dat
Data clean up / create sample data¶
In [ ]:
# data filter
# raw data is from http://www.forextester.com/data/datasources
usdjpy =pd.read_csv('raw_data/USDJPY.txt')
usdjpy = clean_up_df(usdjpy)
#usdjpy.to_csv('new_usdjpy.csv')
import os
foldername = "raw_data/"
for file in os.listdir(foldername):
filename = os.path.splitext(file)[0]
print file, filename
dat = pd.read_csv('raw_data/' + file)
dat = clean_up_df(dat)
dat.to_csv('data/' + filename + '.csv')
In [3]:
dat = pd.read_csv('data/EURUSD.csv')
EUR = pd.Series(list(dat.rate), index = dat.time)
EUR.plot()
Out[3]:
In [4]:
dat = pd.read_csv('data/AUDUSD.csv')
AUD = pd.Series(list(dat.rate), index = dat.time)
AUD.plot()
Out[4]:
In [5]:
dat = pd.read_csv('data/GBPUSD.csv')
GBP = pd.Series(list(dat.rate), index = dat.time)
GBP.plot()
Out[5]:
In [6]:
dat = pd.read_csv('data/NZDUSD.csv')
NZD= pd.Series(list(dat.rate), index = dat.time)
NZD.plot()
Out[6]:
In [15]:
dat = pd.read_csv('data/CHFUSD.csv')
#dat.rate = [1.0/rate for rate in dat.rate]
#dat.to_csv('data/CHFUSD.csv', index = False)
CHF= pd.Series(list(dat.rate), index = dat.time)
CHF.plot()
Out[15]:
In [14]:
dat = pd.read_csv('data/JPYUSD.csv')
#dat.rate = [1.0/rate for rate in dat.rate]
#dat.to_csv('data/JPYUSD.csv', index = False)
JPY= pd.Series(list(dat.rate), index = dat.time)
JPY.plot()
Out[14]:
read sample data¶
In [32]:
def read_ts(path):
all_ts = {}
for file in os.listdir(path):
if file.split('.')[1] == 'csv':
tmp = path + '/' + file
print tmp
dat = pd.read_csv(path + '/' + file) # get data
currency = file[:3] # get currency
dat['day'] = pd.to_datetime(dat.time) # change string to datatime
dat = dat.set_index('day', drop=True) # use day as index
if 'Unnamed: 0' in dat.columns: del dat['Unnamed: 0']
del dat['time'], dat.index.name, dat['currency']
all_ts[currency] = dat # save in a dictionary
return all_ts
all_ts = read_ts('/Users/schang/Dropbox/codes/2016_timeseries_currency/data')
all_ts
Out[32]:
Visualization¶
In [49]:
from bokeh.io import output_notebook
from bokeh.plotting import figure, show
from bokeh.charts import TimeSeries
#from bokeh.models import Range1d
from IPython.display import display
from subprocess import call
TOOLS = "pan, box_zoom, reset, save, box_select"
output_notebook()
In [62]:
df = merge_ts(all_ts)
df.plot()
Out[62]:
In [110]:
all_ts = read_ts('/Users/schang/Dropbox/codes/2016_timeseries_currency/data')
df = merge_ts(all_ts)
df['Date'] = df.index
df = df.reset_index(drop= True)
df.columns
Out[110]:
In [111]:
p = TimeSeries(df, x= 'Date', legend=True,
title="Exchange rate", ylabel='exchange rates')
show(p)
Out[111]:
Modelling¶
In [48]:
# smoother
from statsmodels.nonparametric.smoothers_lowess import lowess
ts = all_ts['JPY']
ts.plot()
def smooth_ts(ts):
all_ts_smoothed = {}
filtered = lowess(list(ts.rate), range(len(ts)), is_sorted=True, frac=0.05, it=0)
return pd.Series(filtered[:,1], index = ts.index)
ts_smoothed = smooth_ts(ts)
ts_smoothed.plot()
Out[48]:
In [36]:
In [38]:
import statsmodels.api as sm
# pair-wise correlation
plot_corr(all_ts)
Out[38]:
In [39]:
from statsmodels.tsa.vector_ar import var_model
from statsmodels.nonparametric.smoothers_lowess import lowess
df = merge_ts(all_ts)
model = var_model.VAR(df, missing= 'drop')
res = model.fit(5)
In [40]:
res.summary()
Out[40]:
In [41]:
res.plot()
In [43]:
res.plot_forecast(40)
In [ ]: